Class 4 – Data Visualization

Data Collection → Data Cleaning → EDA → Feature Engineering → Modeling → Evaluation → Deployment
                       ↑______________|      ↑___________________|
                     (EDA informs         (EDA guides
                      cleaning)            features)

# Use case: Understanding customer demographics
customer_ages.hist(bins=20)
plt.title('Customer Age Distribution')
# Insight: Bimodal distribution reveals two target segments (25-35 and 50-60)

# Use case: Understanding lending patterns
loan_amounts.hist(bins=50)
# Insight: Right-skewed distribution shows most loans are small, few large ones

# Use case: Identifying service bottlenecks
wait_times.hist(bins=30)
# Insight: Long tail indicates some patients wait excessively long

# Use case: Grading curve analysis
exam_scores.hist(bins=20)
# Insight: Normal distribution validates grading fairness

# Use case: Comparing performance across categories
sns.boxplot(x='category', y='revenue', data=df)
# Insight: Electronics has higher median but more variability

# Use case: Compensation equity analysis
sns.boxplot(x='department', y='salary', data=employees)
# Insight: Engineering salaries have wider range, HR more consistent

# Use case: Quality control across shifts
sns.boxplot(x='shift', y='defect_rate', data=production)
# Insight: Night shift has higher median defects - investigate staffing

# Use case: Market analysis for pricing strategy
sns.boxplot(x='neighborhood', y='price', data=homes)
# Insight: Downtown has highest median, but suburbs have luxury outliers

# Use case: ROI analysis for marketing budget allocation
plt.scatter(df['ad_spend'], df['sales'])
# Insight: Strong positive correlation up to $50K, then diminishing returns

# Use case: Patient risk assessment
plt.scatter(patients['bmi'], patients['blood_pressure'])
# Insight: Clear positive correlation - BMI is risk factor

# Use case: Property valuation model
plt.scatter(homes['sqft'], homes['price'], c=homes['bedrooms'])
# Insight: Linear relationship, but bedrooms add value beyond size

# Use case: Optimizing logistics
plt.scatter(orders['distance'], orders['delivery_days'])
# Insight: Non-linear relationship - local deliveries very fast, then plateaus

# Use case: Investment analysis
df.set_index('date')['stock_price'].plot()
# Insight: Upward trend with seasonal dips in January

# Use case: Monitoring site performance
daily_visitors.plot(figsize=(12, 4))
# Insight: Weekend dips, spike after marketing campaign

# Use case: Supply chain management
inventory_df.plot(x='date', y='units')
# Insight: Stockouts occurring quarterly - adjust reorder points

# Use case: Epidemic tracking
cases_by_date.plot()
# Insight: Multiple waves visible, interventions flatten curves

# Use case: Regional performance comparison
revenue_by_region.plot(kind='barh')
# Insight: West region outperforms, South needs attention

# Use case: Resource allocation for support team
ticket_counts.plot(kind='bar')
# Insight: Password resets are #1 issue - implement self-service

# Use case: Program planning and resource allocation
enrollment.plot(kind='barh', color='steelblue')
# Insight: Business programs growing, liberal arts declining

# Use case: Inventory optimization
top_products.plot(kind='barh')
# Insight: 80/20 rule - top 3 products drive 60% of revenue

# Use case: Staffing optimization for retail
pivot = df.pivot_table(values='sales', index='hour', columns='day_of_week')
sns.heatmap(pivot, annot=True, cmap='YlOrRd')
# Insight: Lunch hours (12-1pm) and weekends busiest - schedule accordingly

# Use case: Diversification analysis
corr_matrix = stocks.corr()
sns.heatmap(corr_matrix, annot=True, cmap='coolwarm')
# Insight: Tech stocks highly correlated - need diversification

# Use case: Production planning
sns.heatmap(utilization_pivot, cmap='Greens')
# Insight: Morning shift underutilized - shift maintenance window

# Use case: Personalization strategy
sns.heatmap(segment_features, cmap='Blues')
# Insight: Young professionals prefer mobile, seniors prefer phone

# Use case: Compensation structure analysis
sns.violinplot(x='level', y='salary', data=employees)
# Insight: Senior level has bimodal distribution - two career tracks

# Use case: Educational intervention effectiveness
sns.violinplot(x='method', y='score', data=students)
# Insight: Method A shows wider distribution - inconsistent results

# Use case: Treatment efficacy comparison
sns.violinplot(x='treatment', y='recovery_days', data=patients)
# Insight: New treatment shows faster median and tighter distribution

# Use case: Pricing strategy development
sns.violinplot(x='segment', y='order_value', data=orders)
# Insight: Premium segment shows fat upper tail - upselling opportunity

# Use case: Feature engineering for pricing model
sns.pairplot(homes[['price', 'sqft', 'bedrooms', 'age', 'distance_to_city']])
# Insight: Sqft and bedrooms correlated - may cause multicollinearity

# Use case: Segmentation analysis
sns.pairplot(customers[['recency', 'frequency', 'monetary', 'tenure']])
# Insight: Frequency and monetary highly correlated - RFM score makes sense

# Use case: Process optimization
sns.pairplot(production[['temperature', 'pressure', 'humidity', 'defect_rate']])
# Insight: Temperature and pressure interact - need combined controls

# Use case: Credit scoring model development
sns.pairplot(applicants[['income', 'debt_ratio', 'credit_score', 'default']], hue='default')
# Insight: Income and debt_ratio separate defaulters well

# Use case: Marketing mix optimization
channel_data.plot(kind='bar', stacked=True)
# Insight: Paid search declining, organic growing - adjust budget

# Use case: Quality improvement prioritization
defects_pivot.plot(kind='bar', stacked=True)
# Insight: Assembly defects dominate Product A - focus training there

# Use case: Business model evolution analysis
revenue_mix.plot(kind='bar', stacked=True)
# Insight: Subscription revenue growing faster than one-time sales

# Use case: Diversity and inclusion tracking
headcount.plot(kind='bar', stacked=True)
# Insight: Engineering lacks diversity - recruitment initiative needed

# Use case: Driver positioning strategy
plt.hexbin(trips['pickup_lat'], trips['pickup_lon'], gridsize=30)
# Insight: High density downtown and airport - surge pricing zones

# Use case: New store location planning
plt.hexbin(customers['latitude'], customers['longitude'], gridsize=20)
# Insight: Underserved area in northwest suburbs - expansion opportunity

# Use case: UI/UX optimization
plt.hexbin(clicks['x_position'], clicks['y_position'])
# Insight: Users focus top-left - place CTAs there

# Use case: Network optimization
plt.hexbin(signals['longitude'], signals['latitude'], C=signals['strength'])
# Insight: Dead zones identified in rural areas

# Use case: Product portfolio contribution analysis
df.pivot_table(values='revenue', index='month', columns='product').cumsum().plot.area()
# Insight: Product A drove early growth, Product B sustaining now

# Use case: Channel performance tracking
traffic_sources.plot.area(stacked=True)
# Insight: Direct traffic stable, social media growing rapidly

# Use case: Warehouse space allocation
inventory_by_category.plot.area()
# Insight: Seasonal items taking increasing space - need dynamic allocation

# Use case: Resource planning
resources_by_role.plot.area(stacked=True)
# Insight: Developer hours peak mid-project - plan hiring accordingly

# Use case: Competitive pricing analysis
for brand in brands:
    df[df['brand']==brand]['price'].plot(kind='kde', label=brand)
# Insight: Brand A positioned premium, Brand B budget - clear segmentation

# Use case: Epidemiological analysis
for condition in conditions:
    patients[patients['condition']==condition]['age'].plot(kind='kde')
# Insight: Condition X affects younger population - different treatment approach

# Use case: Risk-return profile comparison
for asset in portfolio:
    returns[asset].plot(kind='kde', label=asset)
# Insight: Crypto shows fat tails - higher risk than traditional assets

# Use case: Success factor analysis
for grade in ['A', 'B', 'C']:
    students[students['grade']==grade]['study_hours'].plot(kind='kde')
# Insight: A students show bimodal distribution - quality over quantity

# Use case: Treatment effect visualization
sns.stripplot(x='treatment', y='response', data=trial_data, jitter=True)
# Insight: Treatment group shows more high responders but also variability

# Use case: School comparison for enrollment decisions
sns.swarmplot(x='school', y='test_score', data=students)
# Insight: School A has tighter distribution - more consistent quality

# Use case: Draft strategy development
sns.stripplot(x='position', y='rating', data=players)
# Insight: Wide variation in quarterback ratings - premium on elite talent

# Use case: Instrument calibration assessment
sns.swarmplot(x='instrument', y='measurement', data=lab_data)
# Insight: Instrument C shows systematic bias - needs recalibration

# Use case: Regional strategy development
g = sns.FacetGrid(sales_df, col='region', row='store_type')
g.map(plt.plot, 'date', 'sales')
# Insight: Urban stores show different seasonality than suburban

# Use case: Content strategy optimization
g = sns.FacetGrid(engagement, col='platform', col_wrap=3)
g.map(sns.histplot, 'engagement_rate')
# Insight: LinkedIn content performs better on weekdays

# Use case: Best practices identification
g = sns.FacetGrid(production, col='plant', row='machine_type')
g.map(sns.scatterplot, 'speed', 'quality')
# Insight: Plant B achieves better speed-quality tradeoff

# Use case: Quality benchmarking
g = sns.FacetGrid(outcomes, col='hospital', row='specialty')
g.map(sns.boxplot, 'treatment', 'recovery_time')
# Insight: Hospital X excels in cardiology - learn from their protocols

# Import required libraries for data manipulation and visualization
import pandas as pd              # For data loading and manipulation
import matplotlib.pyplot as plt  # For creating plots
import seaborn as sns           # For advanced statistical visualizations

# ==================== STEP 1: LOAD DATA ====================
# Read CSV file into a pandas DataFrame
# DataFrame is a 2D table with rows and columns (like Excel)
df = pd.read_csv('data.csv')

# ==================== STEP 2: INITIAL OVERVIEW ====================
# Get basic information about the dataset
# info() shows: column names, data types, non-null counts, memory usage
print(df.info())

# Get statistical summary of numeric columns
# describe() shows: count, mean, std, min, 25%, 50%, 75%, max
print(df.describe())

# ==================== STEP 3: VISUALIZE MISSING DATA ====================
# Create a heatmap to see patterns in missing data
# Missing values show up in one color, present values in another
plt.figure(figsize=(10, 6))  # Set figure size (width=10, height=6 inches)

# df.isnull() creates a boolean matrix: True where data is missing, False otherwise
# Heatmap makes this visual - helps spot if certain columns/rows have missing patterns
sns.heatmap(df.isnull(),           # Boolean DataFrame of missing values
            cbar=False,             # Don't show color bar (not needed for binary True/False)
            yticklabels=False)      # Hide row labels (too many rows to display)
plt.title('Missing Data Pattern')   # Add descriptive title
plt.show()                          # Display the plot

# ==================== STEP 4: CHECK FOR DUPLICATES ====================
# Identify completely identical rows (all columns match)
# duplicated() returns True for rows that are duplicates of earlier rows
duplicate_count = df.duplicated().sum()  # Sum True values to count duplicates
print(f"Duplicate rows: {duplicate_count}")

# Why check? Duplicates can:
# - Bias statistical analyses
# - Inflate sample size artificially
# - Indicate data collection errors

# ==================== STEP 5: VISUALIZE DISTRIBUTIONS ====================
# Create histograms for ALL numeric columns automatically
# Helps quickly spot: skewness, outliers, unusual patterns
df.hist(figsize=(12, 10),  # Overall figure size for all histograms
        bins=30)            # Number of bins (bars) in each histogram
plt.tight_layout()          # Adjust spacing so titles don't overlap
plt.show()                  # Display all histograms

# What to look for in histograms:
# - Normal (bell curve) vs skewed distributions
# - Unexpected spikes or gaps
# - Impossible values (e.g., negative ages)

# Import necessary libraries
import pandas as pd              # Data manipulation
import matplotlib.pyplot as plt  # Plotting
import seaborn as sns           # Statistical visualizations
import numpy as np              # Numerical operations

# ==================== STEP 1: LOAD DATA ====================
# Read housing data from CSV file
df = pd.read_csv('housing.csv')

# Extract the price column as a pandas Series for analysis
# Series is a 1D array with index labels
price = df['price']

# ==================== STEP 2: CALCULATE BASIC STATISTICS ====================
# These statistics give us a numerical summary before visualizing

# count(): Number of non-null values
print(f"Count: {price.count()}")

# mean(): Average value (sum of all values / count)
# :,.2f formats as number with commas and 2 decimal places
print(f"Mean: ${price.mean():,.2f}")

# median(): Middle value when sorted (50th percentile)
# Less affected by outliers than mean
print(f"Median: ${price.median():,.2f}")

# std(): Standard deviation - measures spread around the mean
# Larger values = more variability in data
print(f"Std Dev: ${price.std():,.2f}")

# min() and max(): Range of values
print(f"Min: ${price.min():,.2f}")
print(f"Max: ${price.max():,.2f}")

# skew(): Measure of asymmetry in distribution
# 0 = symmetric (normal distribution)
# Positive = right-skewed (long tail to the right)
# Negative = left-skewed (long tail to the left)
print(f"Skewness: {price.skew():.2f}")

# ==================== STEP 3: CREATE COMPREHENSIVE VISUALIZATION ====================
# Create a 2x2 grid of subplots to show multiple views
# figsize=(12, 10) sets the overall figure size in inches
fig, axes = plt.subplots(2, 2, figsize=(12, 10))

# --------------- Plot 1: HISTOGRAM (Top Left) ---------------
# Shows the frequency distribution - how many values fall in each range
axes[0, 0].hist(price,              # Data to plot
                bins=50,             # Number of bins (bars) to divide the data into
                edgecolor='black',   # Black outline around bars for clarity
                alpha=0.7)           # Transparency (0=invisible, 1=opaque)

# Add vertical lines to show mean and median for comparison
# axvline = add vertical line across the plot
axes[0, 0].axvline(price.mean(),           # x-position: mean value
                    color='red',            # Red for mean
                    linestyle='--',         # Dashed line style
                    label='Mean')           # Label for legend
axes[0, 0].axvline(price.median(),         # x-position: median value  
                    color='green',          # Green for median
                    linestyle='--',         # Dashed line style
                    label='Median')         # Label for legend

axes[0, 0].set_title('Distribution: Histogram')  # Title for this subplot
axes[0, 0].set_xlabel('Price ($)')               # Label for x-axis
axes[0, 0].legend()                              # Show legend with Mean/Median

# --------------- Plot 2: BOX PLOT (Top Right) ---------------
# Shows quartiles, median, and outliers in a compact format
# Box shows middle 50% of data (IQR = Q3 - Q1)
# Whiskers extend to 1.5*IQR, points beyond are outliers
axes[0, 1].boxplot(price,      # Data to plot
                   vert=True)  # Vertical orientation (True) vs horizontal (False)
axes[0, 1].set_title('Distribution: Box Plot')
axes[0, 1].set_ylabel('Price ($)')

# --------------- Plot 3: KDE - KERNEL DENSITY ESTIMATE (Bottom Left) ---------------
# Smooth, continuous version of a histogram
# Shows probability density - area under curve = 1
# Good for seeing overall shape without worrying about bin width
sns.kdeplot(price,              # Data to plot
            ax=axes[1, 0],      # Which subplot to use
            fill=True)          # Fill area under curve (more visually appealing)
axes[1, 0].set_title('Distribution: KDE')
axes[1, 0].set_xlabel('Price ($)')

# --------------- Plot 4: Q-Q PLOT - QUANTILE-QUANTILE PLOT (Bottom Right) ---------------
# Tests if data follows a normal (Gaussian) distribution
# If data is normal, points will fall on the diagonal reference line
# Deviations from line indicate non-normality
from scipy import stats
stats.probplot(price,              # Data to test
               dist="norm",        # Compare to normal distribution
               plot=axes[1, 1])    # Which subplot to use
axes[1, 1].set_title('Q-Q Plot (Normality Check)')

# ==================== STEP 4: DISPLAY THE PLOTS ====================
# tight_layout() automatically adjusts spacing to prevent overlap
plt.tight_layout()

# show() displays all the plots
plt.show()

# INTERPRETATION TIPS:
# - If Mean > Median: Right-skewed (common for prices, incomes)
# - If Mean < Median: Left-skewed
# - Box plot outliers: Investigate these values
# - Q-Q plot: Points on diagonal = data is approximately normal

import numpy as np
import matplotlib.pyplot as plt

# Original skewed data
data = df['income']  # Right-skewed

fig, axes = plt.subplots(2, 2, figsize=(12, 8))

# Original
axes[0, 0].hist(data, bins=50, edgecolor='black')
axes[0, 0].set_title(f'Original (Skew: {data.skew():.2f})')

# Log transformation
axes[0, 1].hist(np.log(data + 1), bins=50, edgecolor='black')
axes[0, 1].set_title(f'Log (Skew: {np.log(data + 1).skew():.2f})')

# Square root transformation
axes[1, 0].hist(np.sqrt(data), bins=50, edgecolor='black')
axes[1, 0].set_title(f'Square Root (Skew: {np.sqrt(data).skew():.2f})')

# Box-Cox transformation
from scipy import stats
transformed, lambda_param = stats.boxcox(data + 1)
axes[1, 1].hist(transformed, bins=50, edgecolor='black')
axes[1, 1].set_title(f'Box-Cox (λ={lambda_param:.2f})')

plt.tight_layout()
plt.show()

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

# Load data
df = pd.read_csv('customers.csv')
category = df['customer_segment']

# 1. Basic statistics
print(f"Unique categories: {category.nunique()}")
print(f"Most common: {category.mode()[0]}")
print(f"Missing values: {category.isnull().sum()}")
print("\nFrequency table:")
print(category.value_counts())
print("\nProportions:")
print(category.value_counts(normalize=True))

# 2. Visualizations
fig, axes = plt.subplots(1, 3, figsize=(15, 5))

# Count plot (vertical bars)
sns.countplot(y=category, order=category.value_counts().index, ax=axes[0])
axes[0].set_title('Distribution: Count Plot')
axes[0].set_xlabel('Count')

# Pie chart (only if few categories)
if category.nunique() <= 5:
    category.value_counts().plot(kind='pie', ax=axes[1], autopct='%1.1f%%')
    axes[1].set_ylabel('')
    axes[1].set_title('Distribution: Pie Chart')
else:
    axes[1].text(0.5, 0.5, 'Too many categories\nfor pie chart', 
                 ha='center', va='center')
    axes[1].axis('off')

# Bar chart (horizontal) with percentages
counts = category.value_counts()
percentages = counts / counts.sum() * 100
axes[2].barh(range(len(counts)), percentages)
axes[2].set_yticks(range(len(counts)))
axes[2].set_yticklabels(counts.index)
axes[2].set_xlabel('Percentage (%)')
axes[2].set_title('Distribution: Percentage')

# Add value labels
for i, v in enumerate(percentages):
    axes[2].text(v + 1, i, f'{v:.1f}%', va='center')

plt.tight_layout()
plt.show()

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np

data = df['transaction_amount']

fig, axes = plt.subplots(2, 3, figsize=(15, 10))

# 1. Box plot (IQR method)
axes[0, 0].boxplot(data)
axes[0, 0].set_title('Box Plot\n(IQR: Q1-1.5*IQR, Q3+1.5*IQR)')
axes[0, 0].set_ylabel('Amount ($)')

# 2. Histogram with outlier threshold
Q1 = data.quantile(0.25)
Q3 = data.quantile(0.75)
IQR = Q3 - Q1
lower_bound = Q1 - 1.5 * IQR
upper_bound = Q3 + 1.5 * IQR

axes[0, 1].hist(data, bins=50, alpha=0.7)
axes[0, 1].axvline(lower_bound, color='red', linestyle='--', label='Lower bound')
axes[0, 1].axvline(upper_bound, color='red', linestyle='--', label='Upper bound')
axes[0, 1].set_title('Histogram with IQR Bounds')
axes[0, 1].legend()

# 3. Z-score method
z_scores = np.abs((data - data.mean()) / data.std())
axes[0, 2].scatter(range(len(data)), z_scores, alpha=0.5)
axes[0, 2].axhline(3, color='red', linestyle='--', label='|z| = 3')
axes[0, 2].set_title('Z-Score Method')
axes[0, 2].set_ylabel('|Z-score|')
axes[0, 2].legend()

# 4. Percentile method
axes[1, 0].hist(data, bins=50, alpha=0.7)
p1 = data.quantile(0.01)
p99 = data.quantile(0.99)
axes[1, 0].axvline(p1, color='red', linestyle='--', label='1st percentile')
axes[1, 0].axvline(p99, color='red', linestyle='--', label='99th percentile')
axes[1, 0].set_title('Percentile Method (1%, 99%)')
axes[1, 0].legend()

# 5. Scatter plot (index vs value)
axes[1, 1].scatter(range(len(data)), data, alpha=0.5)
axes[1, 1].axhline(upper_bound, color='red', linestyle='--')
axes[1, 1].axhline(lower_bound, color='red', linestyle='--')
axes[1, 1].set_title('Index Plot')
axes[1, 1].set_xlabel('Index')
axes[1, 1].set_ylabel('Amount ($)')

# 6. Log scale view
axes[1, 2].hist(np.log(data + 1), bins=50, alpha=0.7)
axes[1, 2].set_title('Log-Transformed Distribution')
axes[1, 2].set_xlabel('Log(Amount + 1)')

plt.tight_layout()
plt.show()

# Print outlier summary
outliers = data[(data < lower_bound) | (data > upper_bound)]
print(f"Number of outliers (IQR): {len(outliers)} ({len(outliers)/len(data)*100:.2f}%)")
print(f"Number of outliers (Z>3): {(z_scores > 3).sum()}")

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np

# Load data
df = pd.read_csv('housing.csv')

fig, axes = plt.subplots(2, 2, figsize=(14, 12))

# 1. Basic scatter plot
axes[0, 0].scatter(df['sqft'], df['price'], alpha=0.5)
axes[0, 0].set_xlabel('Square Feet')
axes[0, 0].set_ylabel('Price ($)')
axes[0, 0].set_title('Basic Scatter Plot')

# 2. With regression line
sns.regplot(x='sqft', y='price', data=df, ax=axes[0, 1], 
            scatter_kws={'alpha': 0.5})
axes[0, 1].set_title('With Regression Line')

# 3. Color-coded by third variable
scatter = axes[1, 0].scatter(df['sqft'], df['price'], 
                              c=df['bedrooms'], cmap='viridis', alpha=0.6)
axes[1, 0].set_xlabel('Square Feet')
axes[1, 0].set_ylabel('Price ($)')
axes[1, 0].set_title('Color-Coded by Bedrooms')
plt.colorbar(scatter, ax=axes[1, 0], label='Bedrooms')

# 4. With marginal distributions
from scipy import stats
axes[1, 1].scatter(df['sqft'], df['price'], alpha=0.5)
axes[1, 1].set_xlabel('Square Feet')
axes[1, 1].set_ylabel('Price ($)')
# Calculate correlation
corr = df['sqft'].corr(df['price'])
axes[1, 1].set_title(f'Correlation: r = {corr:.3f}')
# Add trend line
z = np.polyfit(df['sqft'], df['price'], 1)
p = np.poly1d(z)
axes[1, 1].plot(df['sqft'], p(df['sqft']), "r--", alpha=0.8, linewidth=2)

plt.tight_layout()
plt.show()

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

# Generate overlapping data
np.random.seed(42)
n = 10000
df = pd.DataFrame({
    'x': np.random.randn(n),
    'y': np.random.randn(n) * 0.5 + np.random.randn(n)
})

fig, axes = plt.subplots(2, 3, figsize=(15, 10))

# 1. Problem: Regular scatter (can't see density)
axes[0, 0].scatter(df['x'], df['y'])
axes[0, 0].set_title('❌ Overplotted\n(Cannot see density)')

# 2. Solution 1: Alpha transparency
axes[0, 1].scatter(df['x'], df['y'], alpha=0.1)
axes[0, 1].set_title('✅ Alpha Transparency\n(Shows density)')

# 3. Solution 2: Smaller points
axes[0, 2].scatter(df['x'], df['y'], s=1, alpha=0.5)
axes[0, 2].set_title('✅ Smaller Points')

# 4. Solution 3: 2D histogram (hexbin)
axes[1, 0].hexbin(df['x'], df['y'], gridsize=30, cmap='Blues')
axes[1, 0].set_title('✅ Hexbin (Shows density)')

# 5. Solution 4: 2D density contour
sns.kdeplot(x=df['x'], y=df['y'], ax=axes[1, 1], fill=True, cmap='Blues')
axes[1, 1].set_title('✅ 2D KDE (Smoothed density)')

# 6. Solution 5: Sample the data
sample = df.sample(1000)
axes[1, 2].scatter(sample['x'], sample['y'], alpha=0.5)
axes[1, 2].set_title(f'✅ Random Sample\n({len(sample)} of {len(df)} points)')

plt.tight_layout()
plt.show()

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

# Load data
df = pd.read_csv('employees.csv')

fig, axes = plt.subplots(2, 3, figsize=(15, 10))

# 1. Side-by-side box plots
sns.boxplot(x='department', y='salary', data=df, ax=axes[0, 0])
axes[0, 0].set_title('Box Plots')
axes[0, 0].tick_params(axis='x', rotation=45)

# 2. Violin plots (shows distribution shape)
sns.violinplot(x='department', y='salary', data=df, ax=axes[0, 1])
axes[0, 1].set_title('Violin Plots\n(Shows distribution shape)')
axes[0, 1].tick_params(axis='x', rotation=45)

# 3. Strip plot (shows individual points)
sns.stripplot(x='department', y='salary', data=df, alpha=0.5, ax=axes[0, 2])
axes[0, 2].set_title('Strip Plot\n(Shows all points)')
axes[0, 2].tick_params(axis='x', rotation=45)

# 4. Swarm plot (no overlap)
if len(df) < 1000:  # Only for smaller datasets
    sns.swarmplot(x='department', y='salary', data=df, ax=axes[1, 0])
    axes[1, 0].set_title('Swarm Plot\n(No overlap)')
    axes[1, 0].tick_params(axis='x', rotation=45)
else:
    axes[1, 0].text(0.5, 0.5, 'Too many points\nfor swarm plot', 
                    ha='center', va='center', transform=axes[1, 0].transAxes)

# 5. Overlapping violin + strip
sns.violinplot(x='department', y='salary', data=df, inner=None, ax=axes[1, 1])
sns.stripplot(x='department', y='salary', data=df, color='black', 
              alpha=0.3, size=2, ax=axes[1, 1])
axes[1, 1].set_title('Violin + Strip\n(Best of both)')
axes[1, 1].tick_params(axis='x', rotation=45)

# 6. Histogram overlay
for dept in df['department'].unique():
    data = df[df['department'] == dept]['salary']
    axes[1, 2].hist(data, alpha=0.5, label=dept, bins=20)
axes[1, 2].set_xlabel('Salary')
axes[1, 2].set_ylabel('Count')
axes[1, 2].set_title('Overlapping Histograms')
axes[1, 2].legend()

plt.tight_layout()
plt.show()

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

# Load data
df = pd.read_csv('customers.csv')

# Create cross-tabulation
crosstab = pd.crosstab(df['product_category'], df['customer_segment'])
print("Frequency Table:")
print(crosstab)
print("\nProportions (by row):")
print(pd.crosstab(df['product_category'], df['customer_segment'], normalize='index'))

fig, axes = plt.subplots(2, 2, figsize=(14, 12))

# 1. Grouped bar chart
crosstab.plot(kind='bar', ax=axes[0, 0])
axes[0, 0].set_title('Grouped Bar Chart\n(Absolute counts)')
axes[0, 0].set_xlabel('Product Category')
axes[0, 0].set_ylabel('Count')
axes[0, 0].legend(title='Customer Segment')
axes[0, 0].tick_params(axis='x', rotation=45)

# 2. Stacked bar chart
crosstab.plot(kind='bar', stacked=True, ax=axes[0, 1])
axes[0, 1].set_title('Stacked Bar Chart\n(Shows composition)')
axes[0, 1].set_xlabel('Product Category')
axes[0, 1].set_ylabel('Count')
axes[0, 1].legend(title='Customer Segment')
axes[0, 1].tick_params(axis='x', rotation=45)

# 3. Heatmap (proportions)
crosstab_pct = pd.crosstab(df['product_category'], df['customer_segment'], normalize='index')
sns.heatmap(crosstab_pct, annot=True, fmt='.2%', cmap='YlOrRd', ax=axes[1, 0])
axes[1, 0].set_title('Heatmap\n(Proportions by row)')
axes[1, 0].set_xlabel('Customer Segment')
axes[1, 0].set_ylabel('Product Category')

# 4. Mosaic-style (proportional bars)
crosstab_norm = crosstab.div(crosstab.sum(axis=1), axis=0)
crosstab_norm.plot(kind='barh', stacked=True, ax=axes[1, 1])
axes[1, 1].set_title('Normalized Stacked Bars\n(100% stacked)')
axes[1, 1].set_xlabel('Proportion')
axes[1, 1].set_ylabel('Product Category')
axes[1, 1].legend(title='Customer Segment', bbox_to_anchor=(1.05, 1))

plt.tight_layout()
plt.show()

import pandas as pd
import numpy as np

# Calculate correlation matrix
df = pd.read_csv('data.csv')
corr_matrix = df.corr()

print("Correlation Matrix:")
print(corr_matrix)

# Find highly correlated pairs
threshold = 0.7
high_corr = []
for i in range(len(corr_matrix.columns)):
    for j in range(i+1, len(corr_matrix.columns)):
        if abs(corr_matrix.iloc[i, j]) > threshold:
            high_corr.append((corr_matrix.columns[i], 
                             corr_matrix.columns[j], 
                             corr_matrix.iloc[i, j]))

print(f"\nHigh correlations (|r| > {threshold}):")
for var1, var2, corr in high_corr:
    print(f"{var1} <-> {var2}: r = {corr:.3f}")

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np

# Load data
df = pd.read_csv('sales_data.csv')

# Select numeric columns only
numeric_cols = df.select_dtypes(include=[np.number]).columns
df_numeric = df[numeric_cols]

fig, axes = plt.subplots(1, 3, figsize=(18, 5))

# 1. Basic heatmap
sns.heatmap(df_numeric.corr(), annot=True, fmt='.2f', 
            cmap='coolwarm', center=0, ax=axes[0])
axes[0].set_title('Basic Correlation Heatmap')

# 2. Mask upper triangle (avoid redundancy)
mask = np.triu(np.ones_like(df_numeric.corr(), dtype=bool))
sns.heatmap(df_numeric.corr(), mask=mask, annot=True, fmt='.2f',
            cmap='coolwarm', center=0, square=True, ax=axes[1])
axes[1].set_title('Lower Triangle Only\n(Removes redundancy)')

# 3. Only show strong correlations (|r| > 0.5)
corr = df_numeric.corr()
corr_filtered = corr.copy()
corr_filtered[abs(corr) < 0.5] = 0
sns.heatmap(corr_filtered, annot=True, fmt='.2f',
            cmap='coolwarm', center=0, ax=axes[2])
axes[2].set_title('Strong Correlations Only\n(|r| > 0.5)')

plt.tight_layout()
plt.show()

import matplotlib.pyplot as plt
import seaborn as sns

# Anscombe's quartet (built into seaborn)
anscombe = sns.load_dataset("anscombe")

# Create 2x2 grid
g = sns.FacetGrid(anscombe, col="dataset", col_wrap=2, height=4)
g.map_dataframe(sns.scatterplot, x="x", y="y", s=100)
g.map_dataframe(sns.regplot, x="x", y="y", scatter=False, color='red')

# Add titles showing statistics
for ax, dataset in zip(g.axes.flat, ['I', 'II', 'III', 'IV']):
    data = anscombe[anscombe['dataset'] == dataset]
    corr = data['x'].corr(data['y'])
    ax.set_title(f'Dataset {dataset}\nr = {corr:.3f}')

plt.tight_layout()
plt.show()

import seaborn as sns
import matplotlib.pyplot as plt

# Load data
df = pd.read_csv('data.csv')

# Select subset of numeric variables (too many makes it unreadable)
numeric_vars = ['price', 'sqft', 'bedrooms', 'age', 'distance_to_city']
df_subset = df[numeric_vars]

# Create pair plot
sns.pairplot(df_subset, diag_kind='kde', plot_kws={'alpha': 0.6})
plt.suptitle('Pair Plot: All Pairwise Relationships', y=1.02)
plt.tight_layout()
plt.show()

# With color-coding by category
sns.pairplot(df_subset, hue='property_type', diag_kind='kde', 
             plot_kws={'alpha': 0.6})
plt.suptitle('Pair Plot: Color-Coded by Property Type', y=1.02)
plt.tight_layout()
plt.show()

import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd

# Load data
df = pd.read_csv('housing.csv')

fig, axes = plt.subplots(2, 2, figsize=(14, 12))

# 1. Color encoding (3rd variable)
scatter = axes[0, 0].scatter(df['sqft'], df['price'], 
                              c=df['age'], cmap='viridis', 
                              s=50, alpha=0.6)
axes[0, 0].set_xlabel('Square Feet')
axes[0, 0].set_ylabel('Price ($)')
axes[0, 0].set_title('Color = Age')
plt.colorbar(scatter, ax=axes[0, 0], label='Age (years)')

# 2. Size encoding (4th variable!)
scatter = axes[0, 1].scatter(df['sqft'], df['price'],
                              c=df['age'], cmap='viridis',
                              s=df['bedrooms']*20, alpha=0.6)
axes[0, 1].set_xlabel('Square Feet')
axes[0, 1].set_ylabel('Price ($)')
axes[0, 1].set_title('Color = Age, Size = Bedrooms')
plt.colorbar(scatter, ax=axes[0, 1], label='Age (years)')

# 3. Faceting/small multiples (split by category)
for i, location in enumerate(df['location'].unique()[:4]):
    row = i // 2
    col = i % 2
    if row == 1:
        subset = df[df['location'] == location]
        axes[row, col].scatter(subset['sqft'], subset['price'], alpha=0.6)
        axes[row, col].set_xlabel('Square Feet')
        axes[row, col].set_ylabel('Price ($)')
        axes[row, col].set_title(f'Location: {location}')

plt.tight_layout()
plt.show()

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np

# Load data
df = pd.read_csv('data.csv')

fig, axes = plt.subplots(2, 3, figsize=(16, 10))

# 1. Missing data heatmap
sns.heatmap(df.isnull(), cbar=False, yticklabels=False, 
            cmap='viridis', ax=axes[0, 0])
axes[0, 0].set_title('Missing Data Heatmap\n(Yellow = Missing)')
axes[0, 0].set_xlabel('Columns')
axes[0, 0].set_ylabel('Rows')

# 2. Missing data bar plot
missing_counts = df.isnull().sum()
missing_pct = (missing_counts / len(df)) * 100
missing_pct = missing_pct[missing_pct > 0].sort_values(ascending=False)
axes[0, 1].barh(range(len(missing_pct)), missing_pct)
axes[0, 1].set_yticks(range(len(missing_pct)))
axes[0, 1].set_yticklabels(missing_pct.index)
axes[0, 1].set_xlabel('Missing (%)')
axes[0, 1].set_title('Missing Data by Column')
for i, v in enumerate(missing_pct):
    axes[0, 1].text(v + 1, i, f'{v:.1f}%', va='center')

# 3. Missing data correlation
# Which columns tend to be missing together?
missing_corr = df.isnull().corr()
sns.heatmap(missing_corr, annot=True, fmt='.2f', 
            cmap='coolwarm', center=0, ax=axes[0, 2])
axes[0, 2].set_title('Missingness Correlation\n(Do columns miss together?)')

# 4. Rows with missing data
rows_missing = df.isnull().sum(axis=1)
axes[1, 0].hist(rows_missing, bins=range(0, rows_missing.max()+2), 
                edgecolor='black')
axes[1, 0].set_xlabel('Number of Missing Values per Row')
axes[1, 0].set_ylabel('Count of Rows')
axes[1, 0].set_title('Missing Data Distribution Across Rows')

# 5. Missing data by another variable (check for MAR)
if 'age' in df.columns and 'income' in df.columns:
    df_temp = df.copy()
    df_temp['income_missing'] = df['income'].isnull()
    sns.boxplot(x='income_missing', y='age', data=df_temp, ax=axes[1, 1])
    axes[1, 1].set_title('Age Distribution\nby Income Missingness')
    axes[1, 1].set_xlabel('Income is Missing')
else:
    axes[1, 1].text(0.5, 0.5, 'Example: Check if\nmissingness relates\nto other variables',
                    ha='center', va='center', transform=axes[1, 1].transAxes)
    axes[1, 1].axis('off')

# 6. Missing data summary table
missing_summary = pd.DataFrame({
    'Missing_Count': df.isnull().sum(),
    'Missing_Pct': (df.isnull().sum() / len(df)) * 100,
    'Data_Type': df.dtypes
})
missing_summary = missing_summary[missing_summary['Missing_Count'] > 0]
missing_summary = missing_summary.sort_values('Missing_Pct', ascending=False)
axes[1, 2].axis('off')
table = axes[1, 2].table(cellText=missing_summary.values,
                          colLabels=missing_summary.columns,
                          rowLabels=missing_summary.index,
                          cellLoc='center',
                          loc='center')
table.auto_set_font_size(False)
table.set_fontsize(8)
table.scale(1, 1.5)
axes[1, 2].set_title('Missing Data Summary')

plt.tight_layout()
plt.show()

import pandas as pd
import numpy as np

df = pd.read_csv('data.csv')

# Strategy 1: Delete rows (if MCAR and < 5% missing)
missing_pct = df.isnull().sum() / len(df)
if missing_pct.max() < 0.05:
    print("Strategy: Delete rows with missing values")
    df_clean = df.dropna()
    print(f"Rows dropped: {len(df) - len(df_clean)} ({(len(df) - len(df_clean))/len(df)*100:.1f}%)")

# Strategy 2: Delete columns (if > 50% missing)
cols_to_drop = missing_pct[missing_pct > 0.5].index
if len(cols_to_drop) > 0:
    print(f"\nStrategy: Drop columns with > 50% missing: {list(cols_to_drop)}")
    df_clean = df.drop(columns=cols_to_drop)

# Strategy 3: Simple imputation
for col in df.select_dtypes(include=[np.number]).columns:
    if df[col].isnull().sum() > 0:
        if df[col].skew() < 1:  # Symmetric distribution
            impute_value = df[col].mean()
            method = 'mean'
        else:  # Skewed distribution
            impute_value = df[col].median()
            method = 'median'
        df[col].fillna(impute_value, inplace=True)
        print(f"{col}: Imputed with {method} = {impute_value:.2f}")

# Strategy 4: Mode for categorical
for col in df.select_dtypes(include=['object']).columns:
    if df[col].isnull().sum() > 0:
        mode_value = df[col].mode()[0]
        df[col].fillna(mode_value, inplace=True)
        print(f"{col}: Imputed with mode = {mode_value}")

# Strategy 5: Create missing indicator
for col in df.columns:
    if df[col].isnull().sum() > 0:
        df[f'{col}_was_missing'] = df[col].isnull().astype(int)
        print(f"Created indicator: {col}_was_missing")

import pandas as pd
import numpy as np
from sklearn.impute import SimpleImputer, KNNImputer
from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer

# Load data
df = pd.read_csv('data.csv')
numeric_cols = df.select_dtypes(include=[np.number]).columns

# Method 1: Forward/Backward fill (time series)
df_ffill = df[numeric_cols].fillna(method='ffill')  # Use previous value
df_bfill = df[numeric_cols].fillna(method='bfill')  # Use next value

# Method 2: Linear interpolation (time series)
df_interp = df[numeric_cols].interpolate(method='linear')

# Method 3: KNN imputation (use similar observations)
knn_imputer = KNNImputer(n_neighbors=5)
df_knn = pd.DataFrame(
    knn_imputer.fit_transform(df[numeric_cols]),
    columns=numeric_cols
)

# Method 4: Iterative imputation (predict missing from others)
iter_imputer = IterativeImputer(random_state=42)
df_iter = pd.DataFrame(
    iter_imputer.fit_transform(df[numeric_cols]),
    columns=numeric_cols
)

# Compare methods visually
import matplotlib.pyplot as plt

fig, axes = plt.subplots(2, 3, figsize=(15, 10))

col_to_compare = numeric_cols[0]  # Pick one column

# Original with missing
axes[0, 0].hist(df[col_to_compare].dropna(), bins=30, alpha=0.7, 
                edgecolor='black')
axes[0, 0].set_title(f'Original\n({df[col_to_compare].isnull().sum()} missing)')

# Forward fill
axes[0, 1].hist(df_ffill[col_to_compare], bins=30, alpha=0.7, 
                edgecolor='black')
axes[0, 1].set_title('Forward Fill')

# Interpolation
axes[0, 2].hist(df_interp[col_to_compare], bins=30, alpha=0.7, 
                edgecolor='black')
axes[0, 2].set_title('Linear Interpolation')

# Mean
df_mean = df[numeric_cols].fillna(df[numeric_cols].mean())
axes[1, 0].hist(df_mean[col_to_compare], bins=30, alpha=0.7, 
                edgecolor='black')
axes[1, 0].set_title('Mean Imputation')

# KNN
axes[1, 1].hist(df_knn[col_to_compare], bins=30, alpha=0.7, 
                edgecolor='black')
axes[1, 1].set_title('KNN Imputation (k=5)')

# Iterative
axes[1, 2].hist(df_iter[col_to_compare], bins=30, alpha=0.7, 
                edgecolor='black')
axes[1, 2].set_title('Iterative Imputation')

plt.tight_layout()
plt.show()